{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3d0bda31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.025527000427246094,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)fe2c8/.gitattributes",
       "rate": null,
       "total": 1528,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffdb2cb414de4fe2ab3ae8e4196c3643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)fe2c8/.gitattributes:   0%|          | 0.00/1.53k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.026082992553710938,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)_Pooling/config.json",
       "rate": null,
       "total": 200,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e23c15a52e9248f8b97dca7c0264ec19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)_Pooling/config.json:   0%|          | 0.00/200 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0278017520904541,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)18db3fe2c8/README.md",
       "rate": null,
       "total": 179431,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ca863ca7c6042f4b452f52911d2d06b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)18db3fe2c8/README.md:   0%|          | 0.00/179k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.014539957046508789,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)db3fe2c8/config.json",
       "rate": null,
       "total": 694,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99f05dbf51a34074872f1cd1f8bf6893",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)db3fe2c8/config.json:   0%|          | 0.00/694 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.027724027633666992,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading model.safetensors",
       "rate": null,
       "total": 1112201288,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "610b4b735b0d4a4aa530fcffbff442ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.029436826705932617,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)2c8/onnx/config.json",
       "rate": null,
       "total": 686,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0f29fdd5b174184b18263b4923cd5c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)2c8/onnx/config.json:   0%|          | 0.00/686 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.01871490478515625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading model.onnx",
       "rate": null,
       "total": 1110059084,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e62ba96f1ce04dd5978cd164c7bdc6c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.onnx:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.011972904205322266,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)tencepiece.bpe.model",
       "rate": null,
       "total": 5069051,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d389193c9f74fc391f0da49a9db7260",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.024750232696533203,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)cial_tokens_map.json",
       "rate": null,
       "total": 280,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5c2041ab92047e28391c4c1923bfdca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/280 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.01048898696899414,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading tokenizer.json",
       "rate": null,
       "total": 17082660,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6ef91353e59470b9a5464fe4926e689",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02374410629272461,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 418,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "902dd398fddc44c4b34f67ce164a566a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/418 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.04628419876098633,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading pytorch_model.bin",
       "rate": null,
       "total": 1112242989,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b83266dc96d46f588574dd1912d46d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.03437685966491699,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)nce_bert_config.json",
       "rate": null,
       "total": 57,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cea3a93e04704518bce6f6e159dc2d3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)nce_bert_config.json:   0%|          | 0.00/57.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.021386146545410156,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)tencepiece.bpe.model",
       "rate": null,
       "total": 5069051,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8453ca154c0745558f556b3419b662c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.027351856231689453,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)cial_tokens_map.json",
       "rate": null,
       "total": 280,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "718e0cb7bd104d47b55010e6f768bcf1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/280 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.011173725128173828,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading tokenizer.json",
       "rate": null,
       "total": 17082660,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0535f43c54c144848c1466261cf5644f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.01260232925415039,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 418,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fa41aa0b48e4664987db68754ff4b6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/418 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.028349876403808594,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 28,
       "postfix": null,
       "prefix": "Downloading (…)b3fe2c8/modules.json",
       "rate": null,
       "total": 387,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f8cfd03dedf4797a804fa2e0547a164",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)b3fe2c8/modules.json:   0%|          | 0.00/387 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import getpass\n",
    "import textwrap\n",
    "import torch\n",
    "\n",
    "from elasticsearch import Elasticsearch\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "model = SentenceTransformer(\"intfloat/multilingual-e5-base\", device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f253e5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5fe0f041",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'liuxgm.local', 'cluster_name': 'elasticsearch', 'cluster_uuid': 'lM5ZgEQNTkO8phtT5qZAHQ', 'version': {'number': '8.10.0', 'build_flavor': 'default', 'build_type': 'tar', 'build_hash': 'e338da74c79465dfdc204971e600342b0aa87b6b', 'build_date': '2023-09-07T08:16:21.960703010Z', 'build_snapshot': False, 'lucene_version': '9.7.0', 'minimum_wire_compatibility_version': '7.17.0', 'minimum_index_compatibility_version': '7.0.0'}, 'tagline': 'You Know, for Search'}\n"
     ]
    }
   ],
   "source": [
    "ELASTCSEARCH_CERT_PATH = \"/Users/liuxg/elastic/elasticsearch-8.10.0/config/certs/http_ca.crt\"\n",
    " \n",
    "client = Elasticsearch(  ['https://localhost:9200'],\n",
    "    basic_auth = ('elastic', 'vXDWYtL*my3vnKY9zCfL'),\n",
    "    ca_certs = ELASTCSEARCH_CERT_PATH,\n",
    "    verify_certs = True)\n",
    "print(client.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cda36f29",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/kw/pvmj1zgs44l8t32cs8blf6sc0000gn/T/ipykernel_36180/3293026579.py:21: DeprecationWarning: The 'body' parameter is deprecated and will be removed in a future version. Instead use individual parameters.\n",
      "  client.indices.create(index=\"articles\", body=mapping)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'articles'})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the mapping\n",
    "mapping = {\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"language\": {\"type\": \"keyword\"},\n",
    "            \"id\": {\"type\": \"keyword\"},\n",
    "            \"title\": {\"type\": \"text\"},\n",
    "            \"passage\": {\"type\": \"text\"},\n",
    "            \"passage_embedding\": {\n",
    "                \"type\": \"dense_vector\",\n",
    "                \"dims\": 768,\n",
    "                \"index\": \"true\",\n",
    "                \"similarity\": \"cosine\"\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "# Create the index (deleting any existing index)\n",
    "client.indices.delete(index=\"articles\", ignore_unavailable=True)\n",
    "client.indices.create(index=\"articles\", body=mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4810ff95",
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = [\n",
    "    {\n",
    "        \"language\": \"en\",\n",
    "        \"id\": \"1643584#0\",\n",
    "        \"title\": \"Bloor Street\",\n",
    "        \"passage\": \"\"\"Bloor Street is a major east–west residential and commercial thoroughfare in Toronto, Ontario, Canada. Bloor Street runs from the Prince Edward Viaduct, which spans the Don River Valley, westward into Mississauga where it ends at Central Parkway. East of the viaduct, Danforth Avenue continues along the same right-of-way. The street, approximately long, contains a significant cross-sample of Toronto's ethnic communities. It is also home to Toronto's famous shopping street, the Mink Mile.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"en\",\n",
    "        \"id\": \"2190499#0\",\n",
    "        \"title\": \"Elphinstone College\",\n",
    "        \"passage\": \"\"\"Elphinstone College is an institution of higher education affiliated to the University of Mumbai. Established in 1856, it is one of the oldest colleges of the University of Mumbai. It is reputed for producing luminaries like Bal Gangadhar Tilak, Bhim Rao Ambedkar, Virchand Gandhi, Badruddin Tyabji, Pherozshah Mehta, Kashinath Trimbak Telang, Jamsetji Tata and for illustrious professors that includes Dadabhai Naoroji. It is further observed for having played a key role in spread of Western education in the Bombay Presidency.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"en\",\n",
    "        \"id\": \"8881#0\",\n",
    "        \"title\": \"Doctor (title)\",\n",
    "        \"passage\": \"\"\"Doctor is an academic title that originates from the Latin word of the same spelling and meaning. The word is originally an agentive noun of the Latin verb \"\" 'to teach'. It has been used as an academic title in Europe since the 13th century, when the first Doctorates were awarded at the University of Bologna and the University of Paris. Having become established in European universities, this usage spread around the world. Contracted \"Dr\" or \"Dr.\", it is used as a designation for a person who has obtained a Doctorate (e.g. PhD). In many parts of the world it is also used by medical practitioners, regardless of whether or not they hold a doctoral-level degree.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"de\",\n",
    "        \"id\": \"9002#0\",\n",
    "        \"title\": \"Gesundheits- und Krankenpflege\",\n",
    "        \"passage\": \"\"\"Die Gesundheits- und Krankenpflege als Berufsfeld umfasst die Versorgung und Betreuung von Menschen aller Altersgruppen, insbesondere kranke, behinderte und sterbende Erwachsene. Die Gesundheits- und Kinderkrankenpflege hat ihren Schwerpunkt in der Versorgung von Kindern und Jugendlichen. In beiden Fachrichtungen gehört die Verhütung von Krankheiten und Gesunderhaltung zum Aufgabengebiet der professionellen Pflege.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"de\",\n",
    "        \"id\": \"7769762#0\",\n",
    "        \"title\": \"Tourismusregion (Österreich)\",\n",
    "        \"passage\": \"\"\"Unter Tourismusregion versteht man in Österreich die in den Landestourismusgesetzen verankerten Tourismusverbände mehrerer Gemeinden, im weiteren Sinne aller Gebietskörperschaften.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"de\",\n",
    "        \"id\": \"2270104#0\",\n",
    "        \"title\": \"London Wall\",\n",
    "        \"passage\": \"\"\"London Wall ist die strategische Stadtmauer, die die Römer um Londinium gebaut haben, um die Stadt zu schützen, die über den wichtigen Hafen an der Themse verfügte. Bis ins späte Mittelalter hinein bildete diese Stadtmauer die Grenzen von London. Heute ist \"London Wall\" auch der Name einer Straße, die an einem noch bestehenden Abschnitt der Stadtmauer verläuft.\"\"\",\n",
    "    },\n",
    "    {\n",
    "        \"language\": \"de\",\n",
    "        \"id\": \"2270104#1\",\n",
    "        \"title\": \"London Wall\",\n",
    "        \"passage\": \"\"\"Die Mauer wurde Ende des zweiten oder Anfang des dritten Jahrhunderts erbaut, wahrscheinlich zwischen 190 und 225, vermutlich zwischen 200 und 220. Sie entstand somit etwa achtzig Jahre nach dem im Jahr 120 erfolgten Bau der Festung, deren nördliche und westliche Mauern verstärkt und in der Höhe verdoppelt wurden, um einen Teil der neuen Stadtmauer zu bilden. Die Anlage wurde zumindest bis zum Ende des vierten Jahrhunderts weiter ausgebaut. Sie zählt zu den letzten großen Bauprojekten der Römer vor deren Rückzug aus Britannien im Jahr 410.\"\"\",\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a66068e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'errors': False, 'took': 29, 'items': [{'index': {'_index': 'articles', '_id': 'Ub9jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 0, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'Ur9jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 1, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'U79jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 2, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'VL9jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 3, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'Vb9jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 4, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'Vr9jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 5, '_primary_term': 1, 'status': 201}}, {'index': {'_index': 'articles', '_id': 'V79jCIsByaLf0EuTgc0y', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 6, '_primary_term': 1, 'status': 201}}]})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actions = []\n",
    "for article in articles:\n",
    "    actions.append({\"index\": {\"_index\": \"articles\"}})\n",
    "    passage = article[\"passage\"]\n",
    "    passageEmbedding = model.encode(f\"passage: {passage}\").tolist()\n",
    "    article[\"passage_embedding\"] = passageEmbedding\n",
    "    actions.append(article)\n",
    "\n",
    "client.bulk(index=\"articles\", operations=actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0fa9f8e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_response(response):\n",
    "    for hit in response[\"hits\"][\"hits\"]:\n",
    "        score = hit[\"_score\"]\n",
    "        language = hit[\"_source\"][\"language\"]\n",
    "        id = hit[\"_source\"][\"id\"]\n",
    "        title = hit[\"_source\"][\"title\"]\n",
    "        passage = hit[\"_source\"][\"passage\"]\n",
    "        print()\n",
    "        print(f\"ID: {id}\")\n",
    "        print(f\"Language: {language}\")\n",
    "        print(f\"Title: {title}\")\n",
    "        print(f\"Passage: {textwrap.fill(passage, 120)}\")\n",
    "        print(f\"Score: {score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fa8e6024",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query(q, language=None):\n",
    "    knn = {\n",
    "        \"field\": \"passage_embedding\",\n",
    "        \"query_vector\" : model.encode(f\"query: {q}\").tolist(),\n",
    "        \"k\": 2,\n",
    "        \"num_candidates\": 5\n",
    "    }\n",
    "\n",
    "    if language:\n",
    "        knn[\"filter\"] = {\n",
    "            \"term\": {\n",
    "                \"language\": language,\n",
    "            }\n",
    "        }\n",
    "\n",
    "    return client.search(index=\"articles\", knn=knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "066f44c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ID: 9002#0\n",
      "Language: de\n",
      "Title: Gesundheits- und Krankenpflege\n",
      "Passage: Die Gesundheits- und Krankenpflege als Berufsfeld umfasst die Versorgung und Betreuung von Menschen aller Altersgruppen,\n",
      "insbesondere kranke, behinderte und sterbende Erwachsene. Die Gesundheits- und Kinderkrankenpflege hat ihren Schwerpunkt\n",
      "in der Versorgung von Kindern und Jugendlichen. In beiden Fachrichtungen gehört die Verhütung von Krankheiten und\n",
      "Gesunderhaltung zum Aufgabengebiet der professionellen Pflege.\n",
      "Score: 0.8986235\n",
      "\n",
      "ID: 8881#0\n",
      "Language: en\n",
      "Title: Doctor (title)\n",
      "Passage: Doctor is an academic title that originates from the Latin word of the same spelling and meaning. The word is originally\n",
      "an agentive noun of the Latin verb \"\" 'to teach'. It has been used as an academic title in Europe since the 13th\n",
      "century, when the first Doctorates were awarded at the University of Bologna and the University of Paris. Having become\n",
      "established in European universities, this usage spread around the world. Contracted \"Dr\" or \"Dr.\", it is used as a\n",
      "designation for a person who has obtained a Doctorate (e.g. PhD). In many parts of the world it is also used by medical\n",
      "practitioners, regardless of whether or not they hold a doctoral-level degree.\n",
      "Score: 0.8912681\n"
     ]
    }
   ],
   "source": [
    "pretty_response(query(\"health\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "22954f16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ID: 2270104#0\n",
      "Language: de\n",
      "Title: London Wall\n",
      "Passage: London Wall ist die strategische Stadtmauer, die die Römer um Londinium gebaut haben, um die Stadt zu schützen, die über\n",
      "den wichtigen Hafen an der Themse verfügte. Bis ins späte Mittelalter hinein bildete diese Stadtmauer die Grenzen von\n",
      "London. Heute ist \"London Wall\" auch der Name einer Straße, die an einem noch bestehenden Abschnitt der Stadtmauer\n",
      "verläuft.\n",
      "Score: 0.8941857\n",
      "\n",
      "ID: 2270104#1\n",
      "Language: de\n",
      "Title: London Wall\n",
      "Passage: Die Mauer wurde Ende des zweiten oder Anfang des dritten Jahrhunderts erbaut, wahrscheinlich zwischen 190 und 225,\n",
      "vermutlich zwischen 200 und 220. Sie entstand somit etwa achtzig Jahre nach dem im Jahr 120 erfolgten Bau der Festung,\n",
      "deren nördliche und westliche Mauern verstärkt und in der Höhe verdoppelt wurden, um einen Teil der neuen Stadtmauer zu\n",
      "bilden. Die Anlage wurde zumindest bis zum Ende des vierten Jahrhunderts weiter ausgebaut. Sie zählt zu den letzten\n",
      "großen Bauprojekten der Römer vor deren Rückzug aus Britannien im Jahr 410.\n",
      "Score: 0.870095\n"
     ]
    }
   ],
   "source": [
    "pretty_response(query(\"wall\", language=\"de\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "61971a4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ID: 2270104#1\n",
      "Language: de\n",
      "Title: London Wall\n",
      "Passage: Die Mauer wurde Ende des zweiten oder Anfang des dritten Jahrhunderts erbaut, wahrscheinlich zwischen 190 und 225,\n",
      "vermutlich zwischen 200 und 220. Sie entstand somit etwa achtzig Jahre nach dem im Jahr 120 erfolgten Bau der Festung,\n",
      "deren nördliche und westliche Mauern verstärkt und in der Höhe verdoppelt wurden, um einen Teil der neuen Stadtmauer zu\n",
      "bilden. Die Anlage wurde zumindest bis zum Ende des vierten Jahrhunderts weiter ausgebaut. Sie zählt zu den letzten\n",
      "großen Bauprojekten der Römer vor deren Rückzug aus Britannien im Jahr 410.\n",
      "Score: 0.88160384\n",
      "\n",
      "ID: 2270104#0\n",
      "Language: de\n",
      "Title: London Wall\n",
      "Passage: London Wall ist die strategische Stadtmauer, die die Römer um Londinium gebaut haben, um die Stadt zu schützen, die über\n",
      "den wichtigen Hafen an der Themse verfügte. Bis ins späte Mittelalter hinein bildete diese Stadtmauer die Grenzen von\n",
      "London. Heute ist \"London Wall\" auch der Name einer Straße, die an einem noch bestehenden Abschnitt der Stadtmauer\n",
      "verläuft.\n",
      "Score: 0.87613904\n"
     ]
    }
   ],
   "source": [
    "pretty_response(query(\"Mauer\", language=\"de\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c41450ee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
